package com.itheima.spider.news163;

import com.google.gson.Gson;
import com.itheima.spider.dao.NewsDao;
import com.itheima.spider.pojo.News;
import com.itheima.spider.utils.HttpClientUtils;
import com.itheima.spider.utils.IdWorker;
import com.itheima.spider.utils.JedisUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.jdbc.support.JdbcUtils;
import redis.clients.jedis.Jedis;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class News163Spider {
    public static void main(String[] args) throws Exception {
//确定首页url
        List<String> indexUrlList = new ArrayList<String>();
        indexUrlList.add("https://ent.163.com/special/000380VU/newsdata_index.js?callback=data_callback");
        indexUrlList.add("https://ent.163.com/special/000380VU/newsdata_star.js?callback=data_callback");
        indexUrlList.add("https://ent.163.com/special/000380VU/newsdata_movie.js?callback=data_callback");
        indexUrlList.add("https://ent.163.com/special/000380VU/newsdata_tv.js?callback=data_callback");
        indexUrlList.add("https://ent.163.com/special/000380VU/newsdata_show.js?callback=data_callback");
        indexUrlList.add("https://ent.163.com/special/000380VU/newsdata_music.js?callback=data_callback");
        while (!indexUrlList.isEmpty()) {
            String indexUrl = indexUrlList.remove(0);


            //    String indexUrl = "https://ent.163.com/special/000380VU/newsdata_index.js?callback=data_callback";

//发送请求,获取数据
//        String jsonStr = HttpClientUtils.doGet(indexUrl);
//
//        //将一个非标准的json格式数据转成一个标准的json格式
//        jsonStr = parseJson(jsonStr);
//        //  System.out.println(jsonStr);
//
//        //解析数据
//        try {
//            parseNewsJson(jsonStr);
//        } catch (Exception e) {
//            e.printStackTrace();
//        }
            //执行分页的方法
            page(indexUrl);
        }
    }

    //执行分页爬取数据的方法
    private static void page(String indexUrl) throws Exception {
        String page = "02";
        while (true) {
            //发送请求,获取数据
            String jsonStr = HttpClientUtils.doGet(indexUrl);
            if (jsonStr == null) {
                System.out.println("数据获取完成.......");
                break;
            }
            //将一个非标准的json格式数据转成一个标准的json格式
            jsonStr = parseJson(jsonStr);
            //  System.out.println(jsonStr);

            //解析数据,保存数据

            parseNewsJson(jsonStr);
            //获取下一页的url;


            //indexUrl = "https://ent.163.com/special/000380VU/newsdata_index_" + page + ".js?callback=data_callback";
            if (indexUrl.contains("newsdata_index")) {

                indexUrl = "https://ent.163.com/special/000380VU/newsdata_index_" + page + ".js?callback=data_callback";
            }
            if (indexUrl.contains("newsdata_star")) {
                indexUrl = "https://ent.163.com/special/000380VU/newsdata_star_" + page + ".js?callback=data_callback";
            }
            if (indexUrl.contains("newsdata_movie")) {
                indexUrl = "https://ent.163.com/special/000380VU/newsdata_movie_" + page + ".js?callback=data_callback";

            }
            if (indexUrl.contains("newsdata_tv")) {
                indexUrl = "https://ent.163.com/special/000380VU/newsdata_tv_" + page + ".js?callback=data_callback";

            }
            if (indexUrl.contains("newsdata_show")) {

                indexUrl = "https://ent.163.com/special/000380VU/newsdata_show_" + page + ".js?callback=data_callback";
            }
            if (indexUrl.contains("newsdata_music")) {
                indexUrl = "https://ent.163.com/special/000380VU/newsdata_music_" + page + ".js?callback=data_callback";
            }
            int i = Integer.parseInt(page);
            i++;
            if (i < 10) {
                page = "0" + i;
            } else {
                page = i + "";
            }


        }

    }

    //解析新闻列表中的数据
    private static void parseNewsJson(String jsonStr) throws Exception {
        //将json数据转为java的对象
        Gson gson = new Gson();
        List<Map<String, Object>> newsList = gson.fromJson(jsonStr, List.class);
        //遍历集合,获取集合中的元素
        NewsDao newsdao = new NewsDao();
        for (Map<String, Object> newsMap : newsList) {

            //从map中并没有我们想要的全部数据,而所有的数据都在详情页中,所有需要获取详情页中的数据,在map中唯一对我们有用的是详情页url
            String docurl = (String) newsMap.get("docurl");
            Jedis jedis = JedisUtils.getJedis();
            Boolean flag = jedis.sismember("bigdata:spider:news163:docurl", docurl);
            jedis.close();
            if (flag) {
                continue;
            }


            if (docurl.contains("photoview")) {
                continue;
            }
            if (!docurl.contains("ent.163.com")) {
                continue;
            }
            //解析新闻的详情页
            News news = parseItemNews(docurl);
            System.out.println(news);

            //保存数据
            newsdao.saveNews(news);
            jedis = JedisUtils.getJedis();
            jedis.sadd("bigdata:spider:news163:docurl", news.getDocurl());
            jedis.close();

        }


    }

    //解析新闻的详情页
    private static News parseItemNews(String docurl) throws IOException {
        //发送请求获取数据
        String html = HttpClientUtils.doGet(docurl);
        //通过jsoup解析新闻数据的html
        Document document = Jsoup.parse(html);
        //将解析的新闻数据封装到News对象
        News news = new News();
        //新闻的ID
        IdWorker idWorker = new IdWorker(0, 1);
        long id = idWorker.nextId();
        news.setId(id + "");

        //新闻的标题
        Elements titleEl = document.select("#epContentLeft>h1");
        String title = titleEl.text();

        news.setTitle(title);
        //新闻的时间
        // 2019-06-08 10:20:53　来源: <a id="ne_article_source" href="#" target="_blank">网易娱乐</a>
        Elements timeandsourceEl = document.select(".post_time_source");
        String timeandsource = timeandsourceEl.text();
        String[] split = timeandsource.split("　");
        news.setTime(split[0]);
        //新闻的来源
        Elements sourceEl = document.select("#ne_article_source");
        String source = sourceEl.text();
        news.setSource(source);
        //新闻的正文
        Elements textEl = document.select("#endText p");
        String text = textEl.text();
        news.setContent(text);
        //新闻的编辑
        // 责任编辑：陈少杰_b6952
        Elements editorEl = document.select(".ep-editor");
        String editor = editorEl.text();
        editor.substring(editor.indexOf(":") + 1, editor.indexOf("_"));

        news.setEditor(editor);
        //新闻的docurl
        news.setDocurl(docurl);


        //

        return news;


    }

    //将非标准json格式的数据转成标准格式的json数据
    private static String parseJson(String jsonStr) {
        //data_callback([
        int first = jsonStr.indexOf("(");
        int last = jsonStr.indexOf(")");
        //   System.out.println(first + "..." + last);


        return jsonStr.substring(first + 1, last);


    }


}
